Loop retrying when ballooning out, even when the dom0-min-mem setting means

that there is not sufficient memory available at the moment.  Memory may be
about to be freed up by the system after a domain destruction (i.e. the memory
is being scrubbed asynchronously, and will be released soon).

Closes bug #407, bug #429.

Signed-off-by: Ewan Mellor <ewan@xensource.com>

diff --git a/tools/python/xen/xend/balloon.py b/tools/python/xen/xend/balloon.py
index 69cb465c1cdd10ec3509641b8a10a9ac047d8a07..30d2228be884b858de2a5ba102acd72214cf4c9f 100644 (file)
--- a/tools/python/xen/xend/balloon.py
+++ b/tools/python/xen/xend/balloon.py
@@ -30,52 +30,95 @@ from XendError import VmError
 PROC_XEN_BALLOON = "/proc/xen/balloon"
 BALLOON_OUT_SLACK = 1 # MiB.  We need this because the physinfo details are
                       # rounded.
+RETRY_LIMIT = 10
+##
+# The time to sleep between retries grows linearly, using this value (in
+# seconds).  When the system is lightly loaded, memory should be scrubbed and
+# returned to the system very quickly, whereas when it is loaded, the system
+# needs idle time to get the scrubbing done.  This linear growth accommodates
+# such requirements.
+SLEEP_TIME_GROWTH = 0.1
 
 
 def free(required):
     """Balloon out memory from the privileged domain so that there is the
     specified required amount (in KiB) free.
     """
-    
-    xc = xen.lowlevel.xc.xc()
+
+    # We check whether there is enough free memory, and if not, instruct dom0
+    # to balloon out to free some up.  Memory freed by a destroyed domain may
+    # not appear in the free_memory field immediately, because it needs to be
+    # scrubbed before it can be released to the free list, which is done
+    # asynchronously by Xen; ballooning is asynchronous also.  No matter where
+    # we expect the free memory to come from, therefore, we need to wait for
+    # it to become available.
+    #
+    # We are not allowed to balloon below dom0_min_mem, or if dom0_min_mem
+    # is 0, we cannot balloon at all.  Memory can still become available
+    # through a rebooting domain, however.
+    #
+    # Eventually, we time out (presumably because there really isn't enough
+    # free memory).
+    #
+    # We don't want to set the memory target (triggering a watch) when that
+    # has already been done, but we do want to respond to changing memory
+    # usage, so we recheck the required alloc each time around the loop, but
+    # track the last used value so that we don't trigger too many watches.
+
+    need_mem = (required + 1023) / 1024 + BALLOON_OUT_SLACK
+
     xroot = XendRoot.instance()
+    xc = xen.lowlevel.xc.xc()
 
     try:
-        free_mem = xc.physinfo()['free_memory']
-        need_mem = (required + 1023) / 1024 + BALLOON_OUT_SLACK
+        dom0_min_mem = xroot.get_dom0_min_mem()
 
-        log.debug("Balloon: free %d; need %d.", free_mem, need_mem)
-        
-        if free_mem >= need_mem:
-            return
+        retries = 0
+        sleep_time = SLEEP_TIME_GROWTH
+        last_new_alloc = None
+        while retries < RETRY_LIMIT:
+            free_mem = xc.physinfo()['free_memory']
 
-        dom0_min_mem = xroot.get_dom0_min_mem()
-        if dom0_min_mem == 0:
-            raise VmError('Not enough free memory and dom0_min_mem is 0.')
+            if free_mem >= need_mem:
+                log.debug("Balloon: free %d; need %d; done.", free_mem,
+                          need_mem)
+                return
 
-        dom0_alloc = _get_dom0_alloc()
-        dom0_new_alloc = dom0_alloc - (need_mem - free_mem)
-        if dom0_new_alloc < dom0_min_mem:
-            raise VmError(
-                ('I need %d MiB, but dom0_min_mem is %d and shrinking to '
-                 '%d MiB would leave only %d MiB free.') %
-                (need_mem, dom0_min_mem, dom0_min_mem,
-                 free_mem + (dom0_alloc - dom0_min_mem)))
+            if retries == 0:
+                log.debug("Balloon: free %d; need %d.", free_mem, need_mem)
 
-        dom0 = XendDomain.instance().privilegedDomain()
-        dom0.setMemoryTarget(dom0_new_alloc)
+            if dom0_min_mem > 0:
+                dom0_alloc = _get_dom0_alloc()
+                new_alloc = dom0_alloc - (need_mem - free_mem)
 
-        timeout = 20 # 2 sec
-        while timeout > 0:
-            time.sleep(0.1)
+                if (new_alloc >= dom0_min_mem and
+                    new_alloc != last_new_alloc):
+                    log.debug("Balloon: setting dom0 target to %d.",
+                              new_alloc)
+                    dom0 = XendDomain.instance().privilegedDomain()
+                    dom0.setMemoryTarget(new_alloc)
+                    last_new_alloc = new_alloc
+                    # Continue to retry, waiting for ballooning.
 
-            free_mem = xc.physinfo()['free_memory']
-            if free_mem >= need_mem:
-                return
+            time.sleep(sleep_time)
+            retries += 1
+            sleep_time += SLEEP_TIME_GROWTH
 
-            timeout -= 1
+        # Not enough memory; diagnose the problem.
+        if dom0_min_mem == 0:
+            raise VmError(('Not enough free memory and dom0_min_mem is 0, so '
+                           'I cannot release any more.  I need %d MiB but '
+                           'only have %d.') %
+                          (need_mem, free_mem))
+        elif new_alloc >= dom0_min_mem:
+            raise VmError(
+                ('I need %d MiB, but dom0_min_mem is %d and shrinking to '
+                 '%d MiB would leave only %d MiB free.') %
+                (need_mem, dom0_min_mem, dom0_min_mem,
+                 free_mem + dom0_alloc - dom0_min_mem))
+        else:
+            raise VmError('The privileged domain did not balloon!')
 
-        raise VmError('The privileged domain did not balloon!')
     finally:
         del xc